/*
 *  BWPGazetteer.java
 *
 *
 * Copyright (c) Bruno Woltzenlogel Paleo.
 *
 * This file is the source code of a plugin for GATE (see http://gate.ac.uk/), and is free
 * software, licenced under the GNU Library General Public License,
 * Version 2, June1991.
 *
 * A copy of this licence is included in the distribution of GATE in the file
 * licence.html, and is also available at http://gate.ac.uk/gate/licence.html.
 *
 *  Bruno Woltzenlogel Paleo (bruno.wp@gmail.com ; http://bruno-wp.blogspot.com/ ; http://planeta.terra.com.br/arte/lua/), 4/7/2006
 *
 *  $Id: BWPGazetteer.jav,v 1.4 2007/08/22 08:03:43 oana Exp $
 */

package com.hccl.nlp;

import java.util.*;
import gate.*;
import gate.creole.*;
import gate.creole.gazetteer.*;
import gate.util.*;


/**
 * This class is the implementation of the resource LEVENSHTEINGAZETEER.
 */
public class BWPGazetteer extends AbstractGazetteer {

    /** a map of nodes vs gaz lists */
    protected Map listsByNode;

    /** the maximum normalized distance considered to be a match*/
    private int distanceThreshold = 0;
    private Double normalizedDistanceThreshold = new Double(0.3);
    private Boolean useNormalDistance = new Boolean(true);
    private int dthr = 0;

    /** a flag indicating whether overlapping annotations shouldn't be produced.
     *  if it is true, the algorithm annotates only the best match among
     *  the overlaping ones.*/
    private Boolean avoidOverlapingAnnotations = new Boolean(true);

    /** The separator used for gazetteer entry features */
    protected String gazetteerFeatureSeparator;


    /** the costs associated with each string edition operation*/
    private int deleteCost = 1;
    private int insertCost = 1;
    private int substituteCost = 1;

    private static  int [] posArray;

    static boolean init_num=true;


    protected Trie initialState;

    private ArrayList<ArrayList<Integer> > distancesArray;
    private ArrayList<ArrayList<Integer> > insertionsArray;
    private ArrayList<ArrayList<Integer> > deletionsArray;

    private ArrayList<ArrayList<Integer>> belowDistanceArray;

    public void setAvoidOverlapingAnnotations(Boolean avoid) {
        avoidOverlapingAnnotations = avoid;
    }
    public java.lang.Boolean getAvoidOverlapingAnnotations() {
        return avoidOverlapingAnnotations;
    }

    public void setDistanceThreshold(Integer threshold) {
        distanceThreshold = threshold;
    }
    public java.lang.Integer getDistanceThreshold() {
        return distanceThreshold;
    }

    public java.lang.Double getnormalizedDistanceThreshold() {
        return normalizedDistanceThreshold;
    }

    public void setnormalizedDistanceThreshold(Double normalizedDistanceThreshold) {
        this.normalizedDistanceThreshold = normalizedDistanceThreshold;
    }

    public Boolean getUseNormalDistance() {
        return useNormalDistance;
    }

    public void setUseNormalDistance(Boolean useNormalDistance) {
        this.useNormalDistance = useNormalDistance;
    }



    /** Does the actual loading and parsing of the lists. This method must be
     * called before the gazetteer can be used
     */
    public Resource init() throws ResourceInstantiationException{

//
//        if(init_num){
//            byte[] bytes= JNIDemo.instance().encode("aa");
//            JNIDemo.instance().decode(bytes);
//            init_num = false;
//        }




        initialState = new Trie();

        if(listsURL == null){
            throw new ResourceInstantiationException (
                    "No URL provided for gazetteer creation!");
        }
        definition = new LinearDefinition();
        definition.setSeparator(Strings.unescape(gazetteerFeatureSeparator));
        definition.setURL(listsURL);
        definition.load();
        int linesCnt = definition.size();
        listsByNode = definition.loadLists();
        Iterator inodes = definition.iterator();

        int nodeIdx = 0;
        LinearNode node;
        while (inodes.hasNext()) {
            node = (LinearNode) inodes.next();
            fireStatusChanged("Reading " + node.toString());
            fireProgressChanged(++nodeIdx * 100 / linesCnt);
            readList(node,true);
        } // while iline
        fireProcessFinished();
        return this;
    }

    /** Reads one lists (one file) of phrases
     *
     * @param node the node
     * @param add if <b>true</b> will add the phrases found in the list to the ones
     *     recognised by this gazetter, if <b>false</b> the phrases found in the
     *     list will be removed from the list of phrases recognised by this
     *     gazetteer.
     */
    protected void readList(LinearNode node, boolean add)
            throws ResourceInstantiationException{
        String listName, majorType, minorType, languages,annotationType;
        if ( null == node ) {
            throw new ResourceInstantiationException(" LinearNode node is null ");
        }

        listName = node.getList();
        majorType = node.getMajorType();
        minorType = node.getMinorType();
        languages = node.getLanguage();
        annotationType = node.getAnnotationType();
        GazetteerList gazList = (GazetteerList)listsByNode.get(node);
        if (null == gazList) {
            throw new ResourceInstantiationException("gazetteer list not found by node");
        }

        Iterator iline = gazList.iterator();

        // create default lookup for entries with no arbitary features
        Lookup defaultLookup = new Lookup(listName,majorType, minorType, languages,annotationType);
        defaultLookup.list = node.getList();
        if ( null != mappingDefinition){
            MappingNode mnode = mappingDefinition.getNodeByList(defaultLookup.list);
            if (null!=mnode){
                defaultLookup.oClass = mnode.getClassID();
                defaultLookup.ontology = mnode.getOntologyID();
            }
        }//if mapping def

        Lookup lookup;
        String entry;
        while(iline.hasNext()){
            GazetteerNode gazNode = (GazetteerNode)iline.next();
            entry = gazNode.getEntry();

            Map features = gazNode.getFeatureMap();
            if (features == null) {
                lookup = defaultLookup;
            } else {
                // create a new Lookup object with features
                lookup = new Lookup(listName, majorType, minorType, languages,annotationType);
                lookup.list = node.getList();
                if(null != mappingDefinition) {
                    MappingNode mnode = mappingDefinition.getNodeByList(lookup.list);
                    if(null != mnode) {
                        lookup.oClass = mnode.getClassID();
                        lookup.ontology = mnode.getOntologyID();
                    }
                }// if mapping def
                lookup.features = features;
            }

            if(add)addLookup(entry, lookup);
            else removeLookup(entry, lookup);
        }
    } // void readList(String listDesc)

    /**
     * @return the gazetteerFeatureSeparator
     */
    public String getGazetteerFeatureSeparator() {
        return gazetteerFeatureSeparator;
    }

    /**
     * @param gazetteerFeatureSeparator the gazetteerFeatureSeparator to set
     */
    public void setGazetteerFeatureSeparator(String gazetteerFeatureSeparator) {
        this.gazetteerFeatureSeparator = gazetteerFeatureSeparator;
    }

    /**
     * Adds one phrase to the list of phrases recognized by this gazetteer
     *
     * @param text the phrase to be added
     * @param lookup the description of the annotation associated to this phrase
     */
    public void addLookup(String text, Lookup lookup) {

        if(text.length()<=0)
            return;

        int currentChar;
        Trie currentState = initialState;
        Trie nextState;
        Lookup oldLookup;
        boolean isSpace;

        int[] cpArray = codePoints(text);


        for(int i = 0; i< cpArray.length; i++) {
            if(currentState.getMaxWordLen()==null||currentState.getMaxWordLen()<cpArray.length)
                currentState.setMaxWordLen(cpArray.length);
            currentChar = cpArray[i];
            isSpace = Character.isWhitespace(currentChar);
            if(isSpace) currentChar = ' ';
            else currentChar = (caseSensitive.booleanValue()) ?
                    currentChar :
                    Character.toUpperCase(currentChar) ;
            nextState = currentState.next(currentChar);
            if(nextState == null){
                nextState = new Trie();
                currentState.put(currentChar, nextState);
                if(isSpace) currentState.put(' ',nextState);
            }
            currentState = nextState;
        } //for(int i = 0; i< text.length(); i++)

        currentState.addLookup(lookup);
        currentState.addCodeStr(text);

    } // addLookup

    /** Removes one phrase to the list of phrases recognised by this gazetteer
     *
     * @param text the phrase to be removed
     * @param lookup the description of the annotation associated to this phrase
     */
    public void removeLookup(String text, Lookup lookup) {
        int currentChar;
        Trie currentState = initialState;
        Trie nextState;
        Lookup oldLookup;

        int[] cpArray = codePoints(text);

        for(int i = 0; i< cpArray.length; i++) {
            currentChar = cpArray[i];
            if(Character.isWhitespace(currentChar)) currentChar = ' ';
            nextState = currentState.next(currentChar);
            if(nextState == null) return;//nothing to remove
            currentState = nextState;
        } //for(int i = 0; i< text.length(); i++)
        currentState.removeLookup(lookup);


    } // removeLookup


    /**
     * This method runs the gazetteer. It assumes that all the needed parameters
     * are set. If they are not, an exception will be fired.
     */
    public void execute() throws ExecutionException{

        if (wholeWordsOnly && !avoidOverlapingAnnotations) {
            throw new ExecutionException("BWP Gazetteer should not be used with " +
                    "'wholeWordsOnly == true' and  'avoidOverlapingAnnotations == false'. \n" +
                    "Please reconfigure the parameters of BWPGazetteer to:\n " +
                    "'wholeWordsOnly == true' and  'avoidOverlapingAnnotations == true', \n" +
                    "and the results will be the same as expected with \n" +
                    "'wholeWordsOnly == true' and  'avoidOverlapingAnnotations == false'");
        }

        interrupted = false;
        AnnotationSet annotationSet;
        //check the input
        if(document == null) {
            throw new ExecutionException(
                    "No document to process!"
            );
        }
        String content = document.getContent().toString();
        //System.out.println(content);
        if (!caseSensitive) content = content.toUpperCase();

        if(annotationSetName == null ||
                annotationSetName.equals("")) annotationSet = document.getAnnotations();
        else annotationSet = document.getAnnotations(annotationSetName);

        fireStatusChanged("Doing lookup in " +
                document.getName() + "...");

        String text = content + " "; // to avoid out of boundaries problems

        int[] cpArray = codePoints(text);

        annotate(cpArray,annotationSet);

        fireProcessFinished();
        fireStatusChanged("Lookup complete!");
    } // execute

    /**lookup <br>
     * @param singleItem a single string to be looked up by the gazetteer
     * @return set of the Lookups associated with the parameter*/
    public Set lookup(String singleItem) {
        int currentChar;
        Set set = new HashSet();
        Trie currentState = initialState;
        Trie nextState;

        int[] cp = codePoints(singleItem);

        for(int i = 0; i< cp.length; i++) {
            currentChar = cp[i];
            if(Character.isWhitespace(currentChar)) currentChar = ' ';
            nextState = currentState.next(currentChar);
            if(nextState == null) {
                return set;
            }
            currentState = nextState;
        } //for(int i = 0; i< text.length(); i++)
        set = currentState.getLookupSet();
        return set;

    }

    public boolean remove(String singleItem) {
        int currentChar;
        Trie currentState = initialState;
        Trie nextState;
        Lookup oldLookup;

        int[] cp = codePoints(singleItem);

        for(int i = 0; i< cp.length; i++) {
            currentChar = cp[i];
            if(Character.isWhitespace(currentChar)) currentChar = ' ';
            nextState = currentState.next(currentChar);
            if(nextState == null) {
                return false;
            }//nothing to remove
            currentState = nextState;
        } //for(int i = 0; i< text.length(); i++)
        currentState.lookupSet = new HashSet();
        return true;
    }

    public boolean add(String singleItem, Lookup lookup) {
        addLookup(singleItem, lookup);
        return true;
    }


    private int[] codePoints(String input) {
        return codePoints1(input);
    }

    private static int[] codePoints1(String input) {
        int length = Character.codePointCount(input, 0, input.length());
        int word[] = new int[length];
        for (int i = 0, j = 0, cp = 0; i < input.length(); i += Character.charCount(cp)) {
            word[j++] = cp = input.codePointAt(i);
        }
        return word;
    }




    private boolean computeDistance(int[] content, int strCp ,int strPos) {
        int i = strPos;

        if(belowDistanceArray.size()<=i+1) belowDistanceArray.add(new ArrayList<Integer>());
        else belowDistanceArray.set(i+1,new ArrayList<Integer>());

        if(distancesArray.get(0).size()<=i+1) {
            for (int j=0; j<content.length+1;j++){
                distancesArray.get(j).add(new Integer(i+1));
                insertionsArray.get(j).add(new Integer(0));
                deletionsArray.get(j).add(new Integer(0));
            }
        }
        else {
            for (int j=0; j<content.length+1;j++){
                distancesArray.get(j).set(i + 1, i + 1);
                insertionsArray.get(j).set(i + 1, 0);
                deletionsArray.get(j).set(i+1,0);
            }
        }
        insertionsArray.get(0).set(i+1, i+1);

        for (int n=0; n<belowDistanceArray.get(i).size();n++) {
            int j = belowDistanceArray.get(i).get(n);
            if (strCp == content[j]) {
                distancesArray.get(j+1).set(i+1,distancesArray.get(j).get(i));
                deletionsArray.get(j+1).set(i+1,deletionsArray.get(j).get(i));
                insertionsArray.get(j+1).set(i+1,insertionsArray.get(j).get(i));
            }
            else {
                int delete = distancesArray.get(j+1).get(i) + deleteCost;
                int insert = distancesArray.get(j).get(i+1) + insertCost;
                int substitute = distancesArray.get(j).get(i) + substituteCost;

                if (insert<=delete && insert<=substitute) {
                    distancesArray.get(j+1).set(i+1,insert);
                    deletionsArray.get(j+1).set(i+1,deletionsArray.get(j).get(i+1));
                    insertionsArray.get(j+1).set(i+1,insertionsArray.get(j).get(i+1)+1);
                }
                else if (delete<=substitute){
                    distancesArray.get(j+1).set(i+1,delete);
                    deletionsArray.get(j+1).set(i+1,deletionsArray.get(j+1).get(i)+1);
                    insertionsArray.get(j+1).set(i+1,insertionsArray.get(j+1).get(i));
                }
                else {
                    distancesArray.get(j+1).set(i+1,substitute);
                    deletionsArray.get(j+1).set(i+1,deletionsArray.get(j).get(i));
                    insertionsArray.get(j+1).set(i+1,insertionsArray.get(j).get(i));
                }
            }

            if(distancesArray.get(j+1).get(i+1)<=dthr&&j<content.length-1)
                belowDistanceArray.get(i+1).add(j+1);
        }
        return belowDistanceArray.get(i+1).size()>0;
    }


    private void annotateRecursive(int[] content, AnnotationSet annotationSet, Trie currentState, int pos) {
        int chars[] = currentState.getChars();

        for (int i=0;chars!=null&&i<chars.length; i++) {
            int currentChar = chars[i];
            if(useNormalDistance) dthr = (int)(currentState.getMaxWordLen()*normalizedDistanceThreshold);
            if(computeDistance(content, currentChar, pos)){
                Trie nextState = currentState.next(currentChar);
                if (nextState.isFinal()) {
                   putAnnotation(annotationSet, nextState.getLookupSet(), nextState.getCodeStr(), pos + 1, content);
                }
                annotateRecursive(content, annotationSet, nextState, pos+1 );
            }
        }
    }


    /** Does the actual search for a given word in the text and annotates the ocurrences found.
     *
     * @param content the text where the search should be performed
     * @param annotationSet the current set of annotations associated with the text
     */
    private void annotate(int[] content, AnnotationSet annotationSet) {
        if(useNormalDistance) dthr = Integer.MAX_VALUE;
        else dthr = distanceThreshold;

        distancesArray = new ArrayList<ArrayList<Integer>>();
        insertionsArray = new ArrayList<ArrayList<Integer>>();
        deletionsArray = new ArrayList<ArrayList<Integer>>();
        belowDistanceArray = new ArrayList<ArrayList<Integer>>();
        belowDistanceArray.add(new ArrayList<Integer>());

        for (int j=0; j<content.length+1;j++){
            distancesArray.add(new ArrayList<Integer>());
            insertionsArray.add(new ArrayList<Integer>());
            deletionsArray.add(new ArrayList<Integer>());

        }
        for (int j=0; j<content.length;j++){
            belowDistanceArray.get(0).add(j);

        }
        for (int j=0; j<content.length+1;j++){
            distancesArray.get(j).add(new Integer(0));
            insertionsArray.get(j).add(new Integer(0));
            deletionsArray.get(j).add(new Integer(0));
        }

        annotateRecursive(content, annotationSet, initialState, 0);
    }


    private void putAnnotation(AnnotationSet annotationSet, Lookup lookup, String matchedString,
                               int distance, int matchedRegionStart,
                               int matchedRegionEnd) {
        FeatureMap fm = Factory.newFeatureMap();
        fm.put(LOOKUP_MAJOR_TYPE_FEATURE_NAME, lookup.majorType);
        if (null!= lookup.oClass && null!=lookup.ontology){
            fm.put(LOOKUP_CLASS_FEATURE_NAME,lookup.oClass);
            fm.put(LOOKUP_ONTOLOGY_FEATURE_NAME,lookup.ontology);
        }
        if(null != lookup.minorType) {
            fm.put(LOOKUP_MINOR_TYPE_FEATURE_NAME, lookup.minorType);
            if(null != lookup.languages)
                fm.put("language", lookup.languages);
        }
        if(null != lookup.features) fm.putAll(lookup.features);
        fm.put("matchedString", matchedString);
        fm.put("distance", distance);
        fm.put("normalizedDistance", ((double)distance / matchedString.length()));



        try {

            if(matchedRegionEnd-matchedRegionStart>1||distance<=0)
                annotationSet.add(new Long(matchedRegionStart),
                    new Long(matchedRegionEnd),
                    lookup.annotationType,
                    fm);
        } catch(InvalidOffsetException ioe) {
            //throw new LuckyException(ioe.toString());
        }
    }

        private static int codePoint2strlen(int[] content,int pos) {
        if(pos>content.length) pos = content.length;
        int length = 0;
        for (int i = 0; i < pos; i ++) {
            if(Character.isSupplementaryCodePoint(content[i])) length+=2;
            else length++;
        }
        return length;
    }

    private void putAnnotation(AnnotationSet annotationSet, Set lookupSet, String matchedString,
                               int[] content, int distance, int matchedRegionStart,
                               int matchedRegionEnd) {
        matchedRegionStart = codePoint2strlen(content,matchedRegionStart);
        matchedRegionEnd = codePoint2strlen(content,matchedRegionEnd);
//        matchedRegionStart = JNIDemo.instance().calcl(content,matchedRegionStart);
//        matchedRegionEnd = JNIDemo.instance().calcl(content,matchedRegionEnd);


        for (Object lookup:lookupSet){
            putAnnotation(annotationSet,(Lookup)lookup,matchedString,distance,matchedRegionStart,matchedRegionEnd);
        }
    }

    private void putAnnotation(AnnotationSet annotationSet, Set lookupSet, String matchedString,int cpPos,int[] content) {
       // int dthr = (int)(cpPos*normalizedDistanceThreshold);
        int prevPrevDistance = Integer.MAX_VALUE; // stores the distance 2 columns before. this is important to find the minimum distance
        for (int j=0; j<content.length;j++) {
            int distance = distancesArray.get(j+1).get(cpPos);
            int prevDistance = distancesArray.get(j).get(cpPos);
            if (avoidOverlapingAnnotations) {
                if (distance > prevDistance && prevDistance <= prevPrevDistance) { // a minimum was found

                    if (prevDistance <= dthr) {
                        int matchedRegionEnd = j;
                        int matchedRegionStart = (j) -(cpPos+insertionsArray.get(j).get(cpPos)-deletionsArray.get(j).get(cpPos));

                        if (wholeWordsOnly) { //In this case, we must check if "matchedRegionEnd" and "matchedRegionStart" correspond to the end and beginning of a word
                            int startOffset = 0;
                            int endOffset = 0;

                            if (matchedRegionEnd + endOffset < cpPos) {
                                while (!Character.isWhitespace(content[matchedRegionEnd + endOffset])) {
                                    endOffset++;
                                    if (matchedRegionEnd + endOffset == cpPos-1) break;
                                }
                            }
                            if (matchedRegionStart + startOffset > 0) {
                                while (!Character.isWhitespace(content[matchedRegionStart + startOffset - 1])) {
                                    startOffset--;
                                    if (matchedRegionStart + startOffset == 0) break;
                                }
                            }

                            int currentDistance = prevDistance +(endOffset-startOffset)*insertCost;

                            if (currentDistance <= dthr) {
                                putAnnotation(annotationSet, lookupSet, matchedString, content,
                                        currentDistance, matchedRegionStart+startOffset,
                                        matchedRegionEnd+endOffset);
                            }
                        }
                        else { //(!wholeWordsOnly), then we may simply add the annotation
                            if(matchedString.length()*(1-normalizedDistanceThreshold)>prevDistance)
                                putAnnotation(annotationSet, lookupSet, matchedString, content,
                                    prevDistance, matchedRegionStart,
                                    matchedRegionEnd);
                        }
                    }
                }
            } else {
                //System.out.println("Debug");
                if (!wholeWordsOnly) {
                    if (prevDistance <= dthr) {
                        int startOffset = 0;
                        if (prevDistance < dthr) startOffset -= (int)((dthr - prevDistance)/insertCost);
                        int currentDistance = prevDistance - startOffset*insertCost;
                        while (currentDistance <= dthr) { // Match detected. Create annotation.

                            int matchedRegionEnd = j;
                            int matchedRegionStart = startOffset + (j) -(cpPos+
                                    insertionsArray.get(j).get(cpPos)-
                                    deletionsArray.get(j).get(cpPos));

                            putAnnotation(annotationSet, lookupSet, matchedString, content,
                                    currentDistance, matchedRegionStart,
                                    matchedRegionEnd);

                            startOffset++;
                            if (startOffset > 0) currentDistance += deleteCost;
                            else currentDistance -= insertCost;
                        }
                    }
                }
                else {}
            }

            prevPrevDistance = distancesArray.get(j).get(cpPos);

        }

    }

    /*public static void main(String[] args) {

        BWPGazetteer bwpGazetteer = new BWPGazetteer();

        int[] cpArray = bwpGazetteer.codePoints("我喜欢\uD834\uDF06这个字符");

        for (int i:cpArray){
            System.out.println(i);
        }

        for(int i=0;i<cpArray.length;i++)
            System.out.println(JNIDemo.instance().calcl(cpArray,i)+"\t"
                    +codePoint2strlen(cpArray,i));

        byte[] bytes= JNIDemo.instance().encode("aa");
        System.out.println(JNIDemo.instance().decode(bytes));

    }*/

} // class BWPGazetteer

